{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "## 处理重复数据",
   "id": "9159be9bf07e28cf"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-14T06:08:08.722493Z",
     "start_time": "2025-01-14T06:08:08.714094Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "df_obj = pd.DataFrame({'data1' : ['a'] * 4 + ['b'] * 4,\n",
    "                       'data2' : np.random.randint(0, 4, 8)})\n",
    "print(df_obj)\n",
    "\n",
    "print(df_obj.duplicated()) # 检测重复值，返回布尔值，True表示重复，False表示不重复\n",
    "\n",
    "#按照某一行去重\n",
    "print(df_obj[~df_obj.duplicated()] ) #取出不重复行\n",
    "\n",
    "#按照某一列去重\n",
    "print(df_obj.duplicated('data2'))\n",
    "print(df_obj[~df_obj.duplicated('data2')] )\n"
   ],
   "id": "fd7dd8a5f4c8328f",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  data1  data2\n",
      "0     a      3\n",
      "1     a      3\n",
      "2     a      2\n",
      "3     a      0\n",
      "4     b      1\n",
      "5     b      2\n",
      "6     b      0\n",
      "7     b      2\n",
      "0    False\n",
      "1     True\n",
      "2    False\n",
      "3    False\n",
      "4    False\n",
      "5    False\n",
      "6    False\n",
      "7     True\n",
      "dtype: bool\n",
      "  data1  data2\n",
      "0     a      3\n",
      "2     a      2\n",
      "3     a      0\n",
      "4     b      1\n",
      "5     b      2\n",
      "6     b      0\n",
      "0    False\n",
      "1     True\n",
      "2    False\n",
      "3    False\n",
      "4    False\n",
      "5     True\n",
      "6     True\n",
      "7     True\n",
      "dtype: bool\n",
      "  data1  data2\n",
      "0     a      3\n",
      "2     a      2\n",
      "3     a      0\n",
      "4     b      1\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-14T06:10:28.595045Z",
     "start_time": "2025-01-14T06:10:28.587577Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#在pd的duplicated，是认为空值和空值相等的\n",
    "\n",
    "df_obj1 = pd.DataFrame({'data1' :[np.nan] * 4,\n",
    "                       'data2' :list('1235')})\n",
    "print(df_obj1)\n",
    "\n",
    "df_obj1.duplicated('data1')\n",
    "print(df_obj1.drop_duplicates('data1')) #删除重复值"
   ],
   "id": "77ddd39d30eebc42",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   data1 data2\n",
      "0    NaN     1\n",
      "1    NaN     2\n",
      "2    NaN     3\n",
      "3    NaN     5\n",
      "   data1 data2\n",
      "0    NaN     1\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-14T06:11:51.816684Z",
     "start_time": "2025-01-14T06:11:51.810204Z"
    }
   },
   "cell_type": "code",
   "source": [
    "print(df_obj.drop_duplicates())  #删除完全重复的行\n",
    "print(df_obj.drop_duplicates('data2'))  #根据列中的重复值删除行\n",
    "\n",
    "#如果要在原有的df上去重，需要加inplace=True\n"
   ],
   "id": "43f48891711e02a5",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  data1  data2\n",
      "0     a      3\n",
      "2     a      2\n",
      "3     a      0\n",
      "4     b      1\n",
      "5     b      2\n",
      "6     b      0\n",
      "  data1  data2\n",
      "0     a      3\n",
      "2     a      2\n",
      "3     a      0\n",
      "4     b      1\n"
     ]
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-14T06:14:00.100116Z",
     "start_time": "2025-01-14T06:14:00.095134Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#map与applymap一样，但是map只能用于series，applymap只能用于df\n",
    "ser_obj = pd.Series(np.random.randint(0,10,10))  #series 用map\n",
    "print(ser_obj)\n",
    "\n",
    "print(ser_obj.map(lambda x : x ** 2)) #map()函数，将ser_obj中的每个元素都映射到一个新的元素上"
   ],
   "id": "9e60d3812d158055",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    4\n",
      "1    3\n",
      "2    4\n",
      "3    5\n",
      "4    5\n",
      "5    3\n",
      "6    5\n",
      "7    8\n",
      "8    2\n",
      "9    6\n",
      "dtype: int32\n",
      "0    16\n",
      "1     9\n",
      "2    16\n",
      "3    25\n",
      "4    25\n",
      "5     9\n",
      "6    25\n",
      "7    64\n",
      "8     4\n",
      "9    36\n",
      "dtype: int64\n"
     ]
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-14T06:16:44.348413Z",
     "start_time": "2025-01-14T06:16:44.340550Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#异常值手动替换\n",
    "#Series\n",
    "ser_obj=pd.Series(np.arange(10),index=range(3,13))\n",
    "# 单个值替换单个值\n",
    "print(ser_obj.replace(1, -100))\n",
    "print('-' * 20)\n",
    "# 多个值替换一个值\n",
    "print(ser_obj.replace(range(6,9), -100))\n",
    "print('-' * 20)\n",
    "# 多个值替换多个值\n",
    "print(ser_obj.replace([4, 7], [-100, -200]))"
   ],
   "id": "4bb7b92c6dfe0363",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3       0\n",
      "4    -100\n",
      "5       2\n",
      "6       3\n",
      "7       4\n",
      "8       5\n",
      "9       6\n",
      "10      7\n",
      "11      8\n",
      "12      9\n",
      "dtype: int64\n",
      "--------------------\n",
      "3       0\n",
      "4       1\n",
      "5       2\n",
      "6       3\n",
      "7       4\n",
      "8       5\n",
      "9    -100\n",
      "10   -100\n",
      "11   -100\n",
      "12      9\n",
      "dtype: int64\n",
      "--------------------\n",
      "3       0\n",
      "4       1\n",
      "5       2\n",
      "6       3\n",
      "7    -100\n",
      "8       5\n",
      "9       6\n",
      "10   -200\n",
      "11      8\n",
      "12      9\n",
      "dtype: int64\n"
     ]
    }
   ],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-14T06:22:46.287519Z",
     "start_time": "2025-01-14T06:22:46.275984Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#DataFrame\n",
    "df = pd.DataFrame({'A': [0, 1, 2, 3, 4],\n",
    "                   'B': [5, 6, 7, 8, 9],\n",
    "                   'C': ['a', 'b', 'ac', 'd', 'e']})\n",
    "print(df)\n",
    "#正则表达式替换\n",
    "df.replace(to_replace=r'^a', value=100, regex=True) # 正则表达式替换, 所有以a开头的都替换为100, 并返回新的df, 原df不变.regex=True表示使用正则表达式\n",
    "\n",
    "#想使原df也发生变化, 则需要inplace=True"
   ],
   "id": "2a01df9f301600b6",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   A  B   C\n",
      "0  0  5   a\n",
      "1  1  6   b\n",
      "2  2  7  ac\n",
      "3  3  8   d\n",
      "4  4  9   e\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "   A  B    C\n",
       "0  0  5  100\n",
       "1  1  6    b\n",
       "2  2  7  100\n",
       "3  3  8    d\n",
       "4  4  9    e"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>b</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>d</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>e</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 8
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
